import time
from urllib import parse
import scrapy
import re
from scrapy.http import request
from scrapy.utils.trackref import NoneType
from selenium import webdriver
import requests
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
# from urllib.request import quote, unquote
from selenium.webdriver.common.keys import Keys


class MusicpSpider(scrapy.Spider):
    name = 'musicp'
    allowed_domains = ['music.163.com']
    start_urls = ['https://music.163.com/#/discover/toplist']

    def start_requests(self):
        print("init---------------")
        opt = Options()
        opt.add_argument('--no-sandbox')
        opt.add_argument('--disable-gpu')
        opt.add_experimental_option('excludeSwitches', ['enable-automation'])
        opt.add_argument("--disable-dev-usage")
        desired_capabilities = DesiredCapabilities.CHROME  # 修改页面加载策略
        desired_capabilities["pageLoadStrategy"] = "none"

        # opt.add_argument('--headless')
        self.chrome_driver = 'C:/Users/dd/Desktop/chromedriver.exe'
        self.browser = webdriver.Chrome(self.chrome_driver, options=opt)
        self.browser.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
            "source": """
            Object.defineProperty(navigator, 'webdriver', {
            get: () => undefined
            })
        """
        })
        self.browser.set_window_size(1920, 900)
        self.browser.set_page_load_timeout(30)
        return super().start_requests()

    def parse(self, response):
        print(response.xpath(".//div[@id='comment-box']"))
